{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "import codecs\n",
    "\n",
    "from numpy import array\n",
    "import numpy as np\n",
    "from numpy import argmax\n",
    "from collections import Counter\n",
    "data =[]\n",
    "with codecs.open(\"labeled_set_new(2).txt\", 'r', encoding='utf8') as f_in:\n",
    "    papers = f_in.read()\n",
    "    paperlist = papers.split(\"-------------------------------------------\")\n",
    "    del paperlist[-1]\n",
    "    for paper in paperlist:\n",
    "        data.append(paper.split(\"  |  \"))\n",
    "# data[0] = id, data[1] and data[2] = subfield labels, data[3] and data[4] = AI labels, data[5] and data[6] = research type labels, data[7] = abstract\n",
    "f_in.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>SubfieldM</th>\n",
       "      <th>SubfieldA</th>\n",
       "      <th>AIm</th>\n",
       "      <th>AIa</th>\n",
       "      <th>ResearchM</th>\n",
       "      <th>ResearchA</th>\n",
       "      <th>Abstract</th>\n",
       "      <th>Link1</th>\n",
       "      <th>Link2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10.1016/j.jep.2011.08.069</td>\n",
       "      <td>Treatment, Drug Discovery</td>\n",
       "      <td>Treatment</td>\n",
       "      <td>Neural Network</td>\n",
       "      <td>AITechniqueA</td>\n",
       "      <td>Research</td>\n",
       "      <td>Research</td>\n",
       "      <td>Ethnopharmacological relevance:  Danshensu i...</td>\n",
       "      <td>https://linkinghub.elsevier.com/retrieve/pii/S...</td>\n",
       "      <td>\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\\n10.1117/1.3646916</td>\n",
       "      <td>Diagnosis, Prognosis, Treatment</td>\n",
       "      <td>Diagnosis, Prognosis, Treatment</td>\n",
       "      <td>None</td>\n",
       "      <td>AITechniqueA</td>\n",
       "      <td>Research</td>\n",
       "      <td>Research</td>\n",
       "      <td>Biomarkers are indicators of biological proce...</td>\n",
       "      <td>https://doi.org/10.1117/1.3646916</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/pmid...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\\n10.1109/TPAMI.2010.194</td>\n",
       "      <td>None</td>\n",
       "      <td>None, Other</td>\n",
       "      <td>None</td>\n",
       "      <td>AITechniqueA</td>\n",
       "      <td>Research</td>\n",
       "      <td>Research</td>\n",
       "      <td>Projection methods have been used in the anal...</td>\n",
       "      <td>https://dx.doi.org/10.1109/TPAMI.2010.194</td>\n",
       "      <td>\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\\n10.1016/j.neuroimage.2011.05.084</td>\n",
       "      <td>None</td>\n",
       "      <td>None, Other</td>\n",
       "      <td>None</td>\n",
       "      <td>AITechniqueA</td>\n",
       "      <td>Research</td>\n",
       "      <td>Research</td>\n",
       "      <td>In this paper, a model-based analysis method ...</td>\n",
       "      <td>https://linkinghub.elsevier.com/retrieve/pii/S...</td>\n",
       "      <td>\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\\n10.3390/s110404137</td>\n",
       "      <td>Other</td>\n",
       "      <td>Smart Healthcare</td>\n",
       "      <td>None</td>\n",
       "      <td>AITechniqueA</td>\n",
       "      <td>Review</td>\n",
       "      <td>Review</td>\n",
       "      <td>Biology has often been used as a source of in...</td>\n",
       "      <td>http://www.mdpi.com/resolver?pii=s110404137</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pmc/articles/pmid...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    ID                        SubfieldM  \\\n",
       "0            10.1016/j.jep.2011.08.069        Treatment, Drug Discovery   \n",
       "1                  \\n10.1117/1.3646916  Diagnosis, Prognosis, Treatment   \n",
       "2             \\n10.1109/TPAMI.2010.194                             None   \n",
       "3   \\n10.1016/j.neuroimage.2011.05.084                             None   \n",
       "4                 \\n10.3390/s110404137                            Other   \n",
       "\n",
       "                         SubfieldA             AIm           AIa ResearchM  \\\n",
       "0                        Treatment  Neural Network  AITechniqueA  Research   \n",
       "1  Diagnosis, Prognosis, Treatment            None  AITechniqueA  Research   \n",
       "2                      None, Other            None  AITechniqueA  Research   \n",
       "3                      None, Other            None  AITechniqueA  Research   \n",
       "4                 Smart Healthcare            None  AITechniqueA    Review   \n",
       "\n",
       "  ResearchA                                           Abstract  \\\n",
       "0  Research    Ethnopharmacological relevance:  Danshensu i...   \n",
       "1  Research   Biomarkers are indicators of biological proce...   \n",
       "2  Research   Projection methods have been used in the anal...   \n",
       "3  Research   In this paper, a model-based analysis method ...   \n",
       "4    Review   Biology has often been used as a source of in...   \n",
       "\n",
       "                                               Link1  \\\n",
       "0  https://linkinghub.elsevier.com/retrieve/pii/S...   \n",
       "1                  https://doi.org/10.1117/1.3646916   \n",
       "2          https://dx.doi.org/10.1109/TPAMI.2010.194   \n",
       "3  https://linkinghub.elsevier.com/retrieve/pii/S...   \n",
       "4        http://www.mdpi.com/resolver?pii=s110404137   \n",
       "\n",
       "                                               Link2  \n",
       "0                                                 \\n  \n",
       "1  https://www.ncbi.nlm.nih.gov/pmc/articles/pmid...  \n",
       "2                                                 \\n  \n",
       "3                                                 \\n  \n",
       "4  https://www.ncbi.nlm.nih.gov/pmc/articles/pmid...  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame(data, columns = ['ID','SubfieldM','SubfieldA', 'AIm','AIa','ResearchM','ResearchA','Abstract','Link1','Link2','None'])\n",
    "del df['None']\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  Ethnopharmacological relevance:  Danshensu is an active water-soluble component from Salvia Miltiorrhiza, which has been demonstrated holding multiple mechanisms for the regulation of cardiovascular system. However, the relative contribution of danshensu to its multiple cardiovascular activities remains largely unknown. \n",
      "  Aim of the study:  To develop an artificial neural network (NN) model simultaneously characterizing danshensu pharmacokinetics and multiple cardiovascular activities in acute myocardial infarction (AMI) rats. The relationship between danshensu pharmacokinetics (PK) and pharmacodynamics (PD) were evaluated using contribution values. \n",
      "  Materials and methods:  Danshensu was intraperitoneally injected at a single dose of 20mg/kg to AMI rats induced by coronary artery ligation. Plasma levels of danshensu, cardiac troponin T (cTnT), total homocysteine (Hcy) and reduced glutathione (GSH) were quantified. A back-propagation NN model was developed to characterize the PK and PD profiles of danshensu, in which the input variables contained time, area under plasma concentration-time curve (AUC) of danshensu and rat weights (covariate). Relative contribution of input variable to the output neurons was evaluated using neuron connection weights according to Garson's algorithm. The kinetics of contribution values was also compared and was validated using bootstrap resampling method. \n",
      "  Results:  Danshensu exerted significant cTnT-lowering, Hcy- and GSH-elevating effect, and these marker profiles were well captured by the trained NN model. The calculation of relative contributions revealed that the effect of danshensu on the PD marker could be ranked as cTnT&gt;GSH&gt;Hcy, while the effect of AMI disease on the PD marker could be ranked in the following order: cTnT&gt;Hcy&gt;GSH. The activity of transsulfuration pathway was quite obvious under the AMI state. \n",
      "  Conclusion:  NN is a powerful tool linking PK and PD profiles of danshensu with multiple cardioprotective mechanisms, it provides a simple method for identifying and ranking relative contribution to the multiple therapeutic effects of the drug. \n",
      "\n",
      "Treatment, Drug Discovery\n"
     ]
    }
   ],
   "source": [
    "abstracts = df['Abstract'].values\n",
    "labels = df['SubfieldM'].values\n",
    "print(abstracts[0])\n",
    "print(labels[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pre-process the abstract\n",
    "from string import punctuation\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "\n",
    "def abstract_preprocess(abstracts, labels):\n",
    "    \"\"\"\n",
    "    Remove punctuation from review data and replace multiple space with single space.\n",
    "    Map labels from positive/negative to 1/0.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    abstracts: list of abstracts\n",
    "    labels: list of labels\n",
    "    Returns\n",
    "    -------\n",
    "    all_abstracts: abstracts with punctuation removed\n",
    "    all_words: list of all words occurring in the absctracts\n",
    "    labels: use one-hot encoding\n",
    "    \"\"\"\n",
    "    porter = PorterStemmer()\n",
    "    all_words = list()\n",
    "    all_text = []\n",
    "    for text in abstracts:\n",
    "        new_sent=[]\n",
    "        sentences = sent_tokenize(text)\n",
    "        words = list()\n",
    "        for sent in sentences:\n",
    "            words += word_tokenize(sent)\n",
    "        words = [w.lower() for w in words if w not in punctuation]\n",
    "        #words = [porter.stem(w) for w in words]\n",
    "        words = [w for w in words if w not in stopwords.words(\"english\")]\n",
    "        all_words+=words\n",
    "        text = \" \".join(w for w in words)\n",
    "        all_text.append(text)\n",
    "\n",
    "    #one hot encoding for lables\n",
    "    \n",
    "    \n",
    "    from sklearn.preprocessing import LabelEncoder\n",
    "    from sklearn.preprocessing import OneHotEncoder\n",
    "    all_lab = list()\n",
    "    for lab in labels:\n",
    "        all_lab += lab.split(', ')\n",
    "    all_lab = list(set(all_lab))\n",
    "    \n",
    "    label_encoder = LabelEncoder() \n",
    "    integer_encoded = label_encoder.fit_transform(all_lab)\n",
    "    # binary encode\n",
    "    onehot_encoder = OneHotEncoder(sparse=False)\n",
    "    integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)\n",
    "    onehot_encoded = onehot_encoder.fit_transform(integer_encoded)\n",
    "    \n",
    "    label2hot = {}\n",
    "    i=0\n",
    "    for onehot in onehot_encoded:\n",
    "        label2hot[all_lab[i]] = onehot\n",
    "        i+=1\n",
    "        \n",
    "    new_labels=[]\n",
    "    for lab in labels:\n",
    "        lab = lab.split(', ')\n",
    "        new_l=np.zeros(10)\n",
    "        for l in lab:\n",
    "            new_l += label2hot[l]\n",
    "        new_labels.append(new_l)\n",
    "        \n",
    "        \n",
    "    return all_text, all_words, new_labels, label2hot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "processed_abstracts, words, new_labels, label2hot = abstract_preprocess(abstracts,labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ethnopharmacological relevance danshensu active water-soluble component salvia miltiorrhiza demonstrated holding multiple mechanisms regulation cardiovascular system however relative contribution danshensu multiple cardiovascular activities remains largely unknown aim study develop artificial neural network nn model simultaneously characterizing danshensu pharmacokinetics multiple cardiovascular activities acute myocardial infarction ami rats relationship danshensu pharmacokinetics pk pharmacodynamics pd evaluated using contribution values materials methods danshensu intraperitoneally injected single dose 20mg/kg ami rats induced coronary artery ligation plasma levels danshensu cardiac troponin ctnt total homocysteine hcy reduced glutathione gsh quantified back-propagation nn model developed characterize pk pd profiles danshensu input variables contained time area plasma concentration-time curve auc danshensu rat weights covariate relative contribution input variable output neurons evaluated using neuron connection weights according garson 's algorithm kinetics contribution values also compared validated using bootstrap resampling method results danshensu exerted significant ctnt-lowering hcy- gsh-elevating effect marker profiles well captured trained nn model calculation relative contributions revealed effect danshensu pd marker could ranked ctnt gt gsh gt hcy effect ami disease pd marker could ranked following order ctnt gt hcy gt gsh activity transsulfuration pathway quite obvious ami state conclusion nn powerful tool linking pk pd profiles danshensu multiple cardioprotective mechanisms provides simple method identifying ranking relative contribution multiple therapeutic effects drug\n",
      "[0. 1. 0. 0. 0. 0. 0. 0. 0. 1.]\n"
     ]
    }
   ],
   "source": [
    "print(processed_abstracts[0])\n",
    "print(new_labels[0])\n",
    "words = list(set(words))\n",
    "vocab_size = len(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reviews_create_word_ids(all_words):\n",
    "    \"\"\"\n",
    "    Creates a dictionary mapping each word to an unique id.\n",
    "    Parameters\n",
    "    ----------\n",
    "    all_words: list of all words occurring in the data\n",
    "    Returns\n",
    "    -------\n",
    "    dictionary with word as key and corresponding id as value\n",
    "    \"\"\"\n",
    "    count_words = Counter(all_words)\n",
    "    total_words = len(all_words)\n",
    "    sorted_words = count_words.most_common(total_words)\n",
    "    word_ids = {w: i + 1 for i, (w, c) in enumerate(sorted_words)}\n",
    "    return word_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'prediction': 1,\n",
       " 'hg': 2,\n",
       " 'twenty-one': 3,\n",
       " 'incontinence': 4,\n",
       " 'crabs': 5,\n",
       " 'perspective': 6,\n",
       " 'snowy': 7,\n",
       " 'chf': 8,\n",
       " 'subnetworks': 9,\n",
       " 'both-amci': 10,\n",
       " '52': 11,\n",
       " 'aldh+': 12,\n",
       " 'alive': 13,\n",
       " 'inositolphosphorylceramide': 14,\n",
       " 'proline': 15,\n",
       " 'proximal': 16,\n",
       " '0.02-0.10': 17,\n",
       " 'striatum': 18,\n",
       " 'toxicities': 19,\n",
       " 'h.': 20,\n",
       " 'low-noise': 21,\n",
       " '88.5': 22,\n",
       " 'ictal': 23,\n",
       " 'triggers': 24,\n",
       " \"'holistic\": 25,\n",
       " 'swallowing': 26,\n",
       " 'axon': 27,\n",
       " '92.8': 28,\n",
       " 'd4z4': 29,\n",
       " 'olfm2': 30,\n",
       " 'test-retest': 31,\n",
       " 'allows': 32,\n",
       " 'pregnant': 33,\n",
       " 'prime': 34,\n",
       " 'chloroquine': 35,\n",
       " 'timing': 36,\n",
       " 'navigation': 37,\n",
       " 'needs': 38,\n",
       " 'loaded': 39,\n",
       " 'lfia': 40,\n",
       " 'bots': 41,\n",
       " 'captures': 42,\n",
       " 'overarching': 43,\n",
       " 'inducers': 44,\n",
       " '-1.5': 45,\n",
       " 'salicylate-dependent': 46,\n",
       " '//clinicaltrials.gov/ct2/show/nct02359981': 47,\n",
       " 'abnormal': 48,\n",
       " 'kendo': 49,\n",
       " 'vena': 50,\n",
       " 'baltimore': 51,\n",
       " '40': 52,\n",
       " 'neutron': 53,\n",
       " '=0.98': 54,\n",
       " 'parahippocampal': 55,\n",
       " 'subnational-': 56,\n",
       " 'neuron': 57,\n",
       " '2.44': 58,\n",
       " 'instrumental': 59,\n",
       " 'mediation': 60,\n",
       " '32.7': 61,\n",
       " 'preservation': 62,\n",
       " 'mir2199': 63,\n",
       " 'simeprevir': 64,\n",
       " 'small-world-like': 65,\n",
       " '22.1': 66,\n",
       " 'oradj': 67,\n",
       " 'log-rank': 68,\n",
       " 'downregulated': 69,\n",
       " 'bacillus': 70,\n",
       " 'specialist': 71,\n",
       " 'surgeons': 72,\n",
       " 'atlases': 73,\n",
       " 'precursor': 74,\n",
       " 'virus-derived': 75,\n",
       " 'foot': 76,\n",
       " 'surf': 77,\n",
       " '0.908-0.934': 78,\n",
       " 'implantation': 79,\n",
       " 'postural': 80,\n",
       " '88.06': 81,\n",
       " 'collect': 82,\n",
       " 'radiograph': 83,\n",
       " 'm/s': 84,\n",
       " 'testicular': 85,\n",
       " 'abstracted': 86,\n",
       " 'presumed': 87,\n",
       " '0.6': 88,\n",
       " 'feedback': 89,\n",
       " 'prostate-specific': 90,\n",
       " '66': 91,\n",
       " 'seer': 92,\n",
       " 'intelligence-powered': 93,\n",
       " 'squamous': 94,\n",
       " 'catabolize': 95,\n",
       " 'targets': 96,\n",
       " 'ismb': 97,\n",
       " 'fslnets': 98,\n",
       " 'reasoning': 99,\n",
       " 'prosail': 100,\n",
       " '422': 101,\n",
       " 'salvadora': 102,\n",
       " 'speed': 103,\n",
       " 'like': 104,\n",
       " 'population-level': 105,\n",
       " 'sentences': 106,\n",
       " 'offer': 107,\n",
       " 'mix': 108,\n",
       " 'tackle': 109,\n",
       " 'walls': 110,\n",
       " 'solely': 111,\n",
       " '100-to-328-fold': 112,\n",
       " 'u373': 113,\n",
       " 'cell-wall': 114,\n",
       " 'utr_exons': 115,\n",
       " 'milestone': 116,\n",
       " 'evenly': 117,\n",
       " 'single-subject': 118,\n",
       " 'polychronous': 119,\n",
       " 'pzmbt1': 120,\n",
       " '33': 121,\n",
       " 'strips': 122,\n",
       " 'could': 123,\n",
       " '338': 124,\n",
       " 'trematodes': 125,\n",
       " 'n=7389': 126,\n",
       " 'yearn': 127,\n",
       " 'postulated': 128,\n",
       " '2001-2008': 129,\n",
       " 'deformities': 130,\n",
       " 'numerical': 131,\n",
       " 'capsicum': 132,\n",
       " 'den': 133,\n",
       " 'trnas': 134,\n",
       " 'v79': 135,\n",
       " 'intensity': 136,\n",
       " 'deleted': 137,\n",
       " '57-': 138,\n",
       " 'hypoxia-inducible': 139,\n",
       " 'hcv1a': 140,\n",
       " '2-arm': 141,\n",
       " 'exemplars': 142,\n",
       " '0.073': 143,\n",
       " 'core': 144,\n",
       " 'gasless': 145,\n",
       " 'landmark-boosting': 146,\n",
       " 'adopted': 147,\n",
       " 'files': 148,\n",
       " '845': 149,\n",
       " 'pursuing': 150,\n",
       " 'alk-negative': 151,\n",
       " 'atorvastatin': 152,\n",
       " '30s': 153,\n",
       " 'integrated': 154,\n",
       " 'decades': 155,\n",
       " \"'spiking\": 156,\n",
       " 'mariana': 157,\n",
       " 'medial': 158,\n",
       " 'transportation': 159,\n",
       " 'regulator': 160,\n",
       " '2,234': 161,\n",
       " 'formulations': 162,\n",
       " 'journals': 163,\n",
       " 'aggregate': 164,\n",
       " '100,000': 165,\n",
       " 'placebo-controlled': 166,\n",
       " 'austin': 167,\n",
       " 'demonstrate': 168,\n",
       " '.003': 169,\n",
       " '433': 170,\n",
       " 'criticised': 171,\n",
       " 'hja': 172,\n",
       " 'leverages': 173,\n",
       " 'depauperate': 174,\n",
       " 'macro-anatomy': 175,\n",
       " 'matrix-p': 176,\n",
       " 'always': 177,\n",
       " 'visits': 178,\n",
       " 'socialist-communist': 179,\n",
       " '1.3mg/dl': 180,\n",
       " 'uric': 181,\n",
       " 'double': 182,\n",
       " 'pedigree': 183,\n",
       " '2': 184,\n",
       " '297': 185,\n",
       " '0.21': 186,\n",
       " 'conjunction': 187,\n",
       " 'microfibril': 188,\n",
       " 'considering': 189,\n",
       " 'technological': 190,\n",
       " 'grouped': 191,\n",
       " 'irrelevant': 192,\n",
       " 'influences': 193,\n",
       " 'chance': 194,\n",
       " '3.26': 195,\n",
       " 'gene-environment': 196,\n",
       " 'glm': 197,\n",
       " 'stimulators': 198,\n",
       " 'verification': 199,\n",
       " 'climate': 200,\n",
       " 'lymphovascular': 201,\n",
       " 'cartesian': 202,\n",
       " 'meiotic': 203,\n",
       " 'mouse': 204,\n",
       " 'comprise': 205,\n",
       " 'easy-to-operate': 206,\n",
       " 'gradient': 207,\n",
       " '.16': 208,\n",
       " 'rnaseq': 209,\n",
       " 'cddp': 210,\n",
       " '0-57': 211,\n",
       " 'necrotrophic': 212,\n",
       " '1,114': 213,\n",
       " 'smear': 214,\n",
       " 'adducts/10⁸': 215,\n",
       " '18-aa': 216,\n",
       " 'lacking': 217,\n",
       " '4.1h': 218,\n",
       " 'way': 219,\n",
       " 'comparing': 220,\n",
       " 'interneuron': 221,\n",
       " 'delays': 222,\n",
       " 'examine': 223,\n",
       " 'decision': 224,\n",
       " 'anemia': 225,\n",
       " 'precision/personalized': 226,\n",
       " '492': 227,\n",
       " 'propoxyphene': 228,\n",
       " 'scenario': 229,\n",
       " '600': 230,\n",
       " 'exercised': 231,\n",
       " 'covert': 232,\n",
       " 'pop': 233,\n",
       " 'assistants': 234,\n",
       " '-omics': 235,\n",
       " \"d'souza\": 236,\n",
       " 'glycyrrhizin': 237,\n",
       " 'maternal-newborn': 238,\n",
       " '64.66': 239,\n",
       " 'n=271': 240,\n",
       " 'noninjectivity': 241,\n",
       " 'algorithmic': 242,\n",
       " '584': 243,\n",
       " 'zfp161': 244,\n",
       " 'pause': 245,\n",
       " 'urban': 246,\n",
       " 'driesch': 247,\n",
       " 'whole-brain': 248,\n",
       " 'determines': 249,\n",
       " 'exposure': 250,\n",
       " 'translocation': 251,\n",
       " 'et': 252,\n",
       " 'mw': 253,\n",
       " 'promises': 254,\n",
       " 'redundant': 255,\n",
       " 'rods': 256,\n",
       " 'distribution': 257,\n",
       " 'inconsistencies': 258,\n",
       " '718': 259,\n",
       " 'et-pd': 260,\n",
       " 'unfolded': 261,\n",
       " 'mixed-feature': 262,\n",
       " 'handedness-preserving': 263,\n",
       " '51': 264,\n",
       " 'ccsvm-implementation': 265,\n",
       " 'rethink': 266,\n",
       " 'atrial': 267,\n",
       " 'cage': 268,\n",
       " 'viewpoint': 269,\n",
       " '0.424': 270,\n",
       " 'nucleocytoplasmic': 271,\n",
       " 'hat': 272,\n",
       " 'sativus': 273,\n",
       " 'fiber': 274,\n",
       " '0.02-0.07': 275,\n",
       " '0.921': 276,\n",
       " 'accessibility': 277,\n",
       " 'criticism': 278,\n",
       " 'activity-regulated': 279,\n",
       " 'p53-peptide': 280,\n",
       " 'strains': 281,\n",
       " 'omics-based': 282,\n",
       " 'sensitivities': 283,\n",
       " 'vince': 284,\n",
       " 'succeeded': 285,\n",
       " 'suppressors': 286,\n",
       " '3.54-fold': 287,\n",
       " 'best-performing': 288,\n",
       " 'lggs': 289,\n",
       " '1510.1186': 290,\n",
       " '8428': 291,\n",
       " 'ill-posedness': 292,\n",
       " '.446': 293,\n",
       " 'image-and-context-aware': 294,\n",
       " 'orthopedic': 295,\n",
       " 'onset': 296,\n",
       " '5-point': 297,\n",
       " 'imperative': 298,\n",
       " 'idrid': 299,\n",
       " 'resections': 300,\n",
       " 'circrnas': 301,\n",
       " 'reductionism': 302,\n",
       " '3,097': 303,\n",
       " 'n=30': 304,\n",
       " 'amélioration': 305,\n",
       " 'neurobehavioral': 306,\n",
       " 'proceeded': 307,\n",
       " '740': 308,\n",
       " 'algorithms-have': 309,\n",
       " 'computer': 310,\n",
       " 'formalizable': 311,\n",
       " 'laterality': 312,\n",
       " '110-210': 313,\n",
       " 'strictly': 314,\n",
       " 'rays': 315,\n",
       " 'novel_107': 316,\n",
       " 'oleosin': 317,\n",
       " 'multitask': 318,\n",
       " 'pathophysiology': 319,\n",
       " 'emt': 320,\n",
       " 'interested': 321,\n",
       " 'porto': 322,\n",
       " 'template': 323,\n",
       " 'conveyed': 324,\n",
       " 'semi-automatically': 325,\n",
       " 'smoothly': 326,\n",
       " 'nondestructive': 327,\n",
       " 'resistin': 328,\n",
       " 'survivors': 329,\n",
       " 'urothelial': 330,\n",
       " 'peer-support': 331,\n",
       " 'taxon': 332,\n",
       " 'prepilin': 333,\n",
       " '763': 334,\n",
       " '0.92-0.95': 335,\n",
       " 'place': 336,\n",
       " 'deaths': 337,\n",
       " 'bcr': 338,\n",
       " 'found': 339,\n",
       " 'heterogeneous': 340,\n",
       " 'engineered': 341,\n",
       " 'verbal-amci': 342,\n",
       " '11.0': 343,\n",
       " 'système': 344,\n",
       " 'centimetre-sized': 345,\n",
       " 'spontaneous': 346,\n",
       " 'pregnancy': 347,\n",
       " 'melanogaster': 348,\n",
       " 'actor-network': 349,\n",
       " 'optimally': 350,\n",
       " 'algorithm-selected': 351,\n",
       " 'pharmacokinetic': 352,\n",
       " 'mir156a': 353,\n",
       " 'oncological': 354,\n",
       " 'well-being': 355,\n",
       " 'pharmaceutical': 356,\n",
       " 'mixed-effect': 357,\n",
       " 'kidney': 358,\n",
       " 'wpm': 359,\n",
       " 'pes': 360,\n",
       " 'enhances': 361,\n",
       " 'n=135': 362,\n",
       " 'near-infrared': 363,\n",
       " 'appreciated': 364,\n",
       " 'eastern': 365,\n",
       " 'p=0.004': 366,\n",
       " 'anal': 367,\n",
       " 'ecms': 368,\n",
       " 'blpds': 369,\n",
       " 'appearance': 370,\n",
       " 'tumorgraft': 371,\n",
       " 'attracted': 372,\n",
       " 'alginolyticus': 373,\n",
       " 'within-participant': 374,\n",
       " 'elaborate': 375,\n",
       " 'staging-oriented': 376,\n",
       " 'checkpoint': 377,\n",
       " 'co-repressed': 378,\n",
       " 'paralysis': 379,\n",
       " 'ensembling': 380,\n",
       " '2r-myb': 381,\n",
       " 'map2k4': 382,\n",
       " '240': 383,\n",
       " 'assess': 384,\n",
       " 'animals': 385,\n",
       " 'visualize': 386,\n",
       " 'p=0.09': 387,\n",
       " 'gaiix': 388,\n",
       " 'preliminary': 389,\n",
       " 'checkup': 390,\n",
       " 'base': 391,\n",
       " '590': 392,\n",
       " 'gastrostomy': 393,\n",
       " '24-week': 394,\n",
       " 'jurisdictional': 395,\n",
       " 'correctness': 396,\n",
       " 'low-quality': 397,\n",
       " 'ads': 398,\n",
       " 'distinction': 399,\n",
       " 'purpose-built': 400,\n",
       " 'blueprint': 401,\n",
       " '//clinicaltrials.gov/show/nct02051998': 402,\n",
       " 't-test': 403,\n",
       " 'hopkins': 404,\n",
       " '4f': 405,\n",
       " 'amci': 406,\n",
       " 'boost': 407,\n",
       " 'opacity': 408,\n",
       " 'activate': 409,\n",
       " 'phenylpropanoid': 410,\n",
       " 'spike-time-dependent': 411,\n",
       " 'modulators': 412,\n",
       " 'makes': 413,\n",
       " 'senescent': 414,\n",
       " 'disentangle': 415,\n",
       " 'grades': 416,\n",
       " '17.7': 417,\n",
       " 'parasite': 418,\n",
       " '.this': 419,\n",
       " 'refractive': 420,\n",
       " 'artificielle': 421,\n",
       " 'organs': 422,\n",
       " 'algorithms': 423,\n",
       " 'insights': 424,\n",
       " 'prevented': 425,\n",
       " 'frequency-dependent': 426,\n",
       " 'age-': 427,\n",
       " \"'black-box\": 428,\n",
       " 'correction': 429,\n",
       " 'photoperiod': 430,\n",
       " 'nlp': 431,\n",
       " 'powders': 432,\n",
       " 'pgfets': 433,\n",
       " 'designed': 434,\n",
       " 'probing': 435,\n",
       " 'artificially': 436,\n",
       " 'sub-concepts': 437,\n",
       " 'pt3': 438,\n",
       " 'glioblastoma': 439,\n",
       " 'world-wide': 440,\n",
       " 'severely': 441,\n",
       " 'orbitrap': 442,\n",
       " 'augment': 443,\n",
       " 'referred': 444,\n",
       " 'conn': 445,\n",
       " 'cantharidin': 446,\n",
       " 'fifty-four': 447,\n",
       " 'viii': 448,\n",
       " 'driver-assistance': 449,\n",
       " 'rna-sequencing': 450,\n",
       " 'water-filled': 451,\n",
       " 'seat': 452,\n",
       " 'upstream': 453,\n",
       " '141.8': 454,\n",
       " 'neighborhoods': 455,\n",
       " 'false-positive': 456,\n",
       " 'on-line': 457,\n",
       " 'aligning': 458,\n",
       " 'categorisation': 459,\n",
       " 'middle-income': 460,\n",
       " 'hla-drb5': 461,\n",
       " 'plasmon': 462,\n",
       " 'high-affinity': 463,\n",
       " 'glutathione': 464,\n",
       " 'transferrin': 465,\n",
       " 'generator': 466,\n",
       " '100-100': 467,\n",
       " 'tc': 468,\n",
       " 'lacks': 469,\n",
       " '5.2': 470,\n",
       " '7,989': 471,\n",
       " 'cross-validations': 472,\n",
       " 'underappreciated': 473,\n",
       " 'dtmass': 474,\n",
       " 'analogies': 475,\n",
       " 'fractions': 476,\n",
       " 'proliferative': 477,\n",
       " 'nvr-aco': 478,\n",
       " 'deactivated': 479,\n",
       " '19.97': 480,\n",
       " 'heated': 481,\n",
       " 'germinated': 482,\n",
       " 'caching': 483,\n",
       " 'pharmacogenetics': 484,\n",
       " 'compensating': 485,\n",
       " 'euclidean': 486,\n",
       " 'ultrasonic': 487,\n",
       " 'bags': 488,\n",
       " 'disrupt': 489,\n",
       " 'waiting': 490,\n",
       " 'inheritance': 491,\n",
       " 'taxonomy-folksonomy': 492,\n",
       " 'ranked': 493,\n",
       " 'cyp2c8': 494,\n",
       " 'nonalcoholic': 495,\n",
       " 'layer': 496,\n",
       " 'fosl2': 497,\n",
       " 'interval-valued': 498,\n",
       " 'arm': 499,\n",
       " 'sub-challenges': 500,\n",
       " '46.71': 501,\n",
       " 'subwavelength-scale-digitization': 502,\n",
       " '8510': 503,\n",
       " 'tip': 504,\n",
       " 'tubular': 505,\n",
       " 'pt1': 506,\n",
       " '24-hour': 507,\n",
       " 'encapsulates': 508,\n",
       " 'sim': 509,\n",
       " 'malaysian': 510,\n",
       " 'pharmacy': 511,\n",
       " 'exploit': 512,\n",
       " '0.03': 513,\n",
       " 'gse55457': 514,\n",
       " 'gse39040': 515,\n",
       " 'explorer': 516,\n",
       " 'hybridization': 517,\n",
       " 'paramount': 518,\n",
       " 'exams': 519,\n",
       " 'nonlinear': 520,\n",
       " 'analytically': 521,\n",
       " 'experiments': 522,\n",
       " 'australian': 523,\n",
       " 'small-world': 524,\n",
       " 'toxicogenomics': 525,\n",
       " 'redox': 526,\n",
       " 'annuum': 527,\n",
       " 'icis': 528,\n",
       " 'i.e.': 529,\n",
       " 'purpose-': 530,\n",
       " 'research': 531,\n",
       " 'specificity': 532,\n",
       " '63.88': 533,\n",
       " 'diaphragm': 534,\n",
       " 'co-occurring': 535,\n",
       " 'arts': 536,\n",
       " 'cytotoxicant': 537,\n",
       " 'hippo': 538,\n",
       " '1,3': 539,\n",
       " 'dirichlet': 540,\n",
       " 'nodulation': 541,\n",
       " 'wuzhi': 542,\n",
       " 'deceased': 543,\n",
       " 'comparison': 544,\n",
       " 'cr1': 545,\n",
       " 'fitness': 546,\n",
       " 'media': 547,\n",
       " 'ib2-iib': 548,\n",
       " 'gbm': 549,\n",
       " 'breeding': 550,\n",
       " 'dce': 551,\n",
       " 'ckd': 552,\n",
       " 'tlr': 553,\n",
       " 'accomplish': 554,\n",
       " 'function-specific': 555,\n",
       " 'p.arg83cys': 556,\n",
       " '3.71': 557,\n",
       " 'unterschleissheim': 558,\n",
       " 'p=.15': 559,\n",
       " 'tree-based': 560,\n",
       " '2011': 561,\n",
       " 'itss': 562,\n",
       " 'run': 563,\n",
       " 'norms': 564,\n",
       " 'transcriptomics': 565,\n",
       " 'locus': 566,\n",
       " 'strokes': 567,\n",
       " 'pathogen-free': 568,\n",
       " 'sensor': 569,\n",
       " 'excitation': 570,\n",
       " '3.46': 571,\n",
       " '26-30': 572,\n",
       " 'materials': 573,\n",
       " 'metagenomes': 574,\n",
       " 'iaea': 575,\n",
       " 'mie': 576,\n",
       " 'malaysia': 577,\n",
       " 'antigen': 578,\n",
       " '1.77-': 579,\n",
       " 'near': 580,\n",
       " 'snx-2112': 581,\n",
       " 'non-small': 582,\n",
       " '2a': 583,\n",
       " 'agonists': 584,\n",
       " '512': 585,\n",
       " 'denerd': 586,\n",
       " 'ischaemic': 587,\n",
       " 'epidemiological': 588,\n",
       " 'transcatheter': 589,\n",
       " 'dyes': 590,\n",
       " 'prospectively-collected': 591,\n",
       " 'bulldog': 592,\n",
       " 'certain': 593,\n",
       " 'kyoto': 594,\n",
       " 'bibliographic': 595,\n",
       " 'analyzes': 596,\n",
       " 'hypermotor': 597,\n",
       " 'assists': 598,\n",
       " 'hyperparameter': 599,\n",
       " 'fifth': 600,\n",
       " 'infant': 601,\n",
       " 'funding': 602,\n",
       " 'conversely': 603,\n",
       " 'radicals': 604,\n",
       " 'ltc': 605,\n",
       " 'ensuring': 606,\n",
       " 'unusual': 607,\n",
       " 'avoided': 608,\n",
       " '5.31': 609,\n",
       " 'infected': 610,\n",
       " 'ipsc': 611,\n",
       " 'wuhan': 612,\n",
       " 'knn': 613,\n",
       " 'compromise': 614,\n",
       " 'bioactivity': 615,\n",
       " 'hgdp': 616,\n",
       " 'heptavalent': 617,\n",
       " 'oncomine': 618,\n",
       " 'piezoelectric-based': 619,\n",
       " 'radar': 620,\n",
       " 'aldehyde': 621,\n",
       " 'dimension': 622,\n",
       " '6.03': 623,\n",
       " 'unmet': 624,\n",
       " 'rounding': 625,\n",
       " 'circumstances': 626,\n",
       " 'transcontinental': 627,\n",
       " 'photodamage': 628,\n",
       " 'stomach': 629,\n",
       " 'isr': 630,\n",
       " 'atp2a3': 631,\n",
       " 'pressing': 632,\n",
       " 'pursuit': 633,\n",
       " 'glaucoma': 634,\n",
       " 'monosaccharide': 635,\n",
       " 'past': 636,\n",
       " 'email': 637,\n",
       " 'intronic': 638,\n",
       " 'hu': 639,\n",
       " 'contextualize': 640,\n",
       " 'intra-': 641,\n",
       " 'offshoot': 642,\n",
       " '27': 643,\n",
       " 'pathologists': 644,\n",
       " 'adrenal': 645,\n",
       " 'mass-univariate': 646,\n",
       " 'corrects': 647,\n",
       " 'calibration': 648,\n",
       " 'deriving': 649,\n",
       " '±24.4': 650,\n",
       " 'hek293': 651,\n",
       " 'flash': 652,\n",
       " 'st1926-derivative': 653,\n",
       " 'throat': 654,\n",
       " 'technologies': 655,\n",
       " 'mir172c-induced': 656,\n",
       " 'investigator-': 657,\n",
       " '.001': 658,\n",
       " 'coimmunoprecipitated': 659,\n",
       " 'cover': 660,\n",
       " 'a23187-induced': 661,\n",
       " '16': 662,\n",
       " 'dopamine': 663,\n",
       " 'nedd4': 664,\n",
       " 'ihc': 665,\n",
       " 'assembly': 666,\n",
       " 'woman': 667,\n",
       " 'service-national': 668,\n",
       " 'clay': 669,\n",
       " 'micro-holes': 670,\n",
       " 'copper': 671,\n",
       " 'gsh': 672,\n",
       " 'ceramide': 673,\n",
       " 'connective': 674,\n",
       " 'germ': 675,\n",
       " 'technically': 676,\n",
       " 'proceed': 677,\n",
       " 'codon': 678,\n",
       " 'visualized': 679,\n",
       " 'breaks': 680,\n",
       " '79.2': 681,\n",
       " 'satisfying': 682,\n",
       " 'trend': 683,\n",
       " '521': 684,\n",
       " 'grading': 685,\n",
       " 'midwives': 686,\n",
       " 'dsrna': 687,\n",
       " 'integrates': 688,\n",
       " 'working': 689,\n",
       " '0.25-0.47': 690,\n",
       " 'sirnas': 691,\n",
       " 'landmarks': 692,\n",
       " 'erse': 693,\n",
       " 'transitions': 694,\n",
       " '//clinicaltrials.gov/show/nct04330924': 695,\n",
       " 'confer': 696,\n",
       " 'freund': 697,\n",
       " 'emg-pr': 698,\n",
       " 'mirna-hub': 699,\n",
       " 'tumor-infiltrating': 700,\n",
       " 'sage-seq': 701,\n",
       " 'release': 702,\n",
       " 'replication': 703,\n",
       " 'immunosuppression': 704,\n",
       " 'occupations': 705,\n",
       " '94.67': 706,\n",
       " '47.5': 707,\n",
       " 'mixtures': 708,\n",
       " 'streptococcus': 709,\n",
       " 'preferential': 710,\n",
       " 'communication': 711,\n",
       " 'bagging': 712,\n",
       " 'ml/min': 713,\n",
       " 'apetala2': 714,\n",
       " 'bioc': 715,\n",
       " 'diffusion-': 716,\n",
       " 'enter': 717,\n",
       " 'l': 718,\n",
       " 'derzeit': 719,\n",
       " 'customized': 720,\n",
       " '30.7': 721,\n",
       " 'endpoints': 722,\n",
       " 'crispr-cas9': 723,\n",
       " 'cyclophosphamide': 724,\n",
       " 'planning': 725,\n",
       " '18': 726,\n",
       " 'repressed': 727,\n",
       " 'solve': 728,\n",
       " 'disease-free': 729,\n",
       " 'illinois': 730,\n",
       " \"'voice\": 731,\n",
       " 'adjudicated': 732,\n",
       " 'families': 733,\n",
       " 'prometteuses': 734,\n",
       " 'reconstruct': 735,\n",
       " 'anatomically': 736,\n",
       " 'well': 737,\n",
       " 'accelerometer': 738,\n",
       " 'c130': 739,\n",
       " 'extremity': 740,\n",
       " 'labor': 741,\n",
       " 'purpose': 742,\n",
       " 'bromodomain-containing': 743,\n",
       " 'responder': 744,\n",
       " 'reveal': 745,\n",
       " 'absorption': 746,\n",
       " '.043': 747,\n",
       " 'palsy': 748,\n",
       " 'on-chip': 749,\n",
       " 'square': 750,\n",
       " 'signal-transduction': 751,\n",
       " 'blended': 752,\n",
       " 'performed': 753,\n",
       " 'sdr': 754,\n",
       " 'cck8': 755,\n",
       " 'between-country': 756,\n",
       " 'weanling': 757,\n",
       " '10.9': 758,\n",
       " 'm0': 759,\n",
       " 'immunoreactivity': 760,\n",
       " 'dominate': 761,\n",
       " 'considerably': 762,\n",
       " '172': 763,\n",
       " 'sporisorium': 764,\n",
       " 'patterns': 765,\n",
       " 'novelty': 766,\n",
       " 'inefficient': 767,\n",
       " 'vci': 768,\n",
       " \"'attention\": 769,\n",
       " 'anti-tuberculosis': 770,\n",
       " 'causal': 771,\n",
       " '100': 772,\n",
       " 'differs': 773,\n",
       " 'degenerations': 774,\n",
       " 'touches': 775,\n",
       " 'electrons': 776,\n",
       " 'ml⁻¹': 777,\n",
       " '0.87': 778,\n",
       " 'pulvinar': 779,\n",
       " 'proprietary': 780,\n",
       " '90': 781,\n",
       " 'impair': 782,\n",
       " 'k-means': 783,\n",
       " 'occurrences': 784,\n",
       " 'fibrin': 785,\n",
       " 'urgent': 786,\n",
       " 'nr4a1': 787,\n",
       " 'comprehensively': 788,\n",
       " 'clarified': 789,\n",
       " 'consented': 790,\n",
       " 'sampling': 791,\n",
       " 'three-level': 792,\n",
       " 'tomosynthesis': 793,\n",
       " 'hydromorphone': 794,\n",
       " 'mir408': 795,\n",
       " 'afin': 796,\n",
       " 'regimes': 797,\n",
       " 'eci': 798,\n",
       " 'judgments': 799,\n",
       " 'uncertainties': 800,\n",
       " 'sphingolipids': 801,\n",
       " 'confounder': 802,\n",
       " 'controlling': 803,\n",
       " 'worms': 804,\n",
       " 'exploratory': 805,\n",
       " 'conversational': 806,\n",
       " 'lmg': 807,\n",
       " 'junctions': 808,\n",
       " 'low-level': 809,\n",
       " 'mir-29': 810,\n",
       " 'mipred': 811,\n",
       " 'fifteen': 812,\n",
       " 'loocv': 813,\n",
       " 'ict': 814,\n",
       " 'traces': 815,\n",
       " 'urologic': 816,\n",
       " 'institutionally': 817,\n",
       " 'youth': 818,\n",
       " 'univariable': 819,\n",
       " 'aldehydes': 820,\n",
       " 'n=28': 821,\n",
       " 'atherosclerosis': 822,\n",
       " '227': 823,\n",
       " 'anosmic': 824,\n",
       " 'electromagnetic': 825,\n",
       " 'dynamics': 826,\n",
       " 'p=.004': 827,\n",
       " 'secretion': 828,\n",
       " 'accurately': 829,\n",
       " 'von': 830,\n",
       " 'itraq-based': 831,\n",
       " 'l.': 832,\n",
       " 'reconstructed': 833,\n",
       " 'tests': 834,\n",
       " 'blunted': 835,\n",
       " 'paramagnetic': 836,\n",
       " 'sports': 837,\n",
       " 'issue=2': 838,\n",
       " '0.797': 839,\n",
       " 'residue-specific': 840,\n",
       " '//clinicaltrials.gov/ct2/show/nct04330924': 841,\n",
       " 'canada': 842,\n",
       " '288': 843,\n",
       " 'microbiologic': 844,\n",
       " 'abundant': 845,\n",
       " 'answering': 846,\n",
       " 'nerve-sparing': 847,\n",
       " 'compute': 848,\n",
       " 'omitted': 849,\n",
       " 'amyotrophic': 850,\n",
       " 'aur1': 851,\n",
       " 'eg': 852,\n",
       " 'tsp': 853,\n",
       " 'impacted': 854,\n",
       " 'posttest': 855,\n",
       " 'nurd': 856,\n",
       " 'tables': 857,\n",
       " 'existed': 858,\n",
       " 'electrostatic': 859,\n",
       " 'rbi': 860,\n",
       " 'qualities': 861,\n",
       " 'intrapartum': 862,\n",
       " 'transform': 863,\n",
       " 'ighv1-02': 864,\n",
       " '55+': 865,\n",
       " 'increments': 866,\n",
       " 'innovatus': 867,\n",
       " 'occasionally': 868,\n",
       " 'sb-ce': 869,\n",
       " '318': 870,\n",
       " '098': 871,\n",
       " 'modern': 872,\n",
       " 'epithelium': 873,\n",
       " 'delay': 874,\n",
       " 'stimulated': 875,\n",
       " 'nfe2l2': 876,\n",
       " 'circuitries': 877,\n",
       " 'autopsy': 878,\n",
       " 'approximating': 879,\n",
       " '2013': 880,\n",
       " 'amyloid-positive': 881,\n",
       " 'effets': 882,\n",
       " 'biofortification': 883,\n",
       " '31-90': 884,\n",
       " 'pvy': 885,\n",
       " 'srna': 886,\n",
       " 'picea': 887,\n",
       " 'replace': 888,\n",
       " 'f-score': 889,\n",
       " 'pd': 890,\n",
       " 'transcription-semi-quantitative': 891,\n",
       " 'ankle': 892,\n",
       " 'isobaric': 893,\n",
       " 'tetra-': 894,\n",
       " 'primordium': 895,\n",
       " 'gene-encoded': 896,\n",
       " 'target-reaching': 897,\n",
       " 'multi-sensor': 898,\n",
       " '//linkinghub.elsevier.com/retrieve/pii/s1053-8119': 899,\n",
       " 'deformed': 900,\n",
       " 'β-cells': 901,\n",
       " 'p=0.013': 902,\n",
       " 'memory-related': 903,\n",
       " 'october': 904,\n",
       " 'mirp': 905,\n",
       " '2-12': 906,\n",
       " '0.83-0.88': 907,\n",
       " 'crystallized': 908,\n",
       " 'hp1/ddb1': 909,\n",
       " 'subject': 910,\n",
       " 'differentiate': 911,\n",
       " 'truncatula': 912,\n",
       " 'patterning': 913,\n",
       " 'gbaf': 914,\n",
       " 'periodically': 915,\n",
       " 'specialized': 916,\n",
       " 'gaining': 917,\n",
       " '0.80': 918,\n",
       " '17,319': 919,\n",
       " 'svm': 920,\n",
       " 'polyprotein': 921,\n",
       " 'dose-dependently': 922,\n",
       " 'ring': 923,\n",
       " 'tbi': 924,\n",
       " 'otx1': 925,\n",
       " '52.5': 926,\n",
       " 'modulates': 927,\n",
       " 'co-purifications': 928,\n",
       " 'pte': 929,\n",
       " 'insall-salvati': 930,\n",
       " 'importance': 931,\n",
       " 'alternative': 932,\n",
       " 'rwe': 933,\n",
       " 'contracted': 934,\n",
       " 'normally': 935,\n",
       " 'triglyceride': 936,\n",
       " 'deformations': 937,\n",
       " 'fbj': 938,\n",
       " 'thinkers': 939,\n",
       " 'tides': 940,\n",
       " 'quickly': 941,\n",
       " 'force': 942,\n",
       " 'necrotrophy': 943,\n",
       " 'conduction': 944,\n",
       " 'glycolytic': 945,\n",
       " 'gercor': 946,\n",
       " 'proto‑oncogene': 947,\n",
       " 'gac': 948,\n",
       " 'pk': 949,\n",
       " 'inevitable': 950,\n",
       " 'convenience': 951,\n",
       " 'raw': 952,\n",
       " 'drought-prone': 953,\n",
       " 'categories': 954,\n",
       " 'surrounding': 955,\n",
       " 'auteurs': 956,\n",
       " 'approving': 957,\n",
       " 'bemisia': 958,\n",
       " 'starch': 959,\n",
       " 'age-weighted': 960,\n",
       " 'asknivi': 961,\n",
       " 'spain': 962,\n",
       " 'han': 963,\n",
       " 'discontinuation': 964,\n",
       " 'ezh2': 965,\n",
       " 'fundus-controlled': 966,\n",
       " 'ɛ-greedy': 967,\n",
       " 'justified': 968,\n",
       " 'radishes': 969,\n",
       " 'thermodynamics': 970,\n",
       " 'screens': 971,\n",
       " 'matters': 972,\n",
       " 'respiration': 973,\n",
       " '75.8±8.4': 974,\n",
       " 'relevant': 975,\n",
       " 'mitochondria-encoded': 976,\n",
       " 'simple': 977,\n",
       " 'outline': 978,\n",
       " 'cytostatic': 979,\n",
       " 'practitioners': 980,\n",
       " 'ht': 981,\n",
       " 'exacerbations': 982,\n",
       " 'key': 983,\n",
       " 'assessing': 984,\n",
       " 'pharyngo-laryngeal': 985,\n",
       " 'j-s': 986,\n",
       " 'attach': 987,\n",
       " 'sequence-view': 988,\n",
       " 'bud': 989,\n",
       " 'dosage': 990,\n",
       " 'prefecture': 991,\n",
       " '1.021': 992,\n",
       " 'valves': 993,\n",
       " 'wild-type': 994,\n",
       " 'display': 995,\n",
       " 'pfs': 996,\n",
       " 'imagination': 997,\n",
       " 'compte': 998,\n",
       " 'life-': 999,\n",
       " 'u.s.': 1000,\n",
       " ...}"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_ids = reviews_create_word_ids(words)\n",
    "word_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def reviews_encode(word_ids, abstracts):\n",
    "    \"\"\"\n",
    "    Replace each word in the review with its corresponding id specified in the\n",
    "    dictionary word_ids\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    word_ids: dictionary with word as key and id as value\n",
    "    abstracts: abstract data\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    abstract data with each word replaced by its corresponding id\n",
    "\n",
    "    \"\"\"\n",
    "    encoded_abstracts = list()\n",
    "    for abstract in abstracts:\n",
    "        encoded_abstract = list()\n",
    "        for word in abstract.split():\n",
    "            if word not in word_ids.keys():\n",
    "                # if word is not available in word_ids put 0 in that place\n",
    "                encoded_abstract.append(0)\n",
    "            else:\n",
    "                encoded_abstract.append(word_ids[word])\n",
    "        encoded_abstracts.append(encoded_abstract)\n",
    "\n",
    "    return encoded_abstracts\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1715"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_abstracts = reviews_encode(word_ids, processed_abstracts)\n",
    "len(processed_abstracts[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "138"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(processed_abstracts[1].split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[5255,\n",
       " 11430,\n",
       " 18245,\n",
       " 8277,\n",
       " 10852,\n",
       " 3597,\n",
       " 9893,\n",
       " 12003,\n",
       " 7599,\n",
       " 15556,\n",
       " 9474,\n",
       " 340,\n",
       " 8560,\n",
       " 17163,\n",
       " 5850,\n",
       " 16064,\n",
       " 641,\n",
       " 16498,\n",
       " 7749,\n",
       " 15858,\n",
       " 12350,\n",
       " 11027,\n",
       " 13936,\n",
       " 2362,\n",
       " 13230,\n",
       " 4597,\n",
       " 8311,\n",
       " 14531,\n",
       " 17644,\n",
       " 4096,\n",
       " 7651,\n",
       " 18041,\n",
       " 7980,\n",
       " 17673,\n",
       " 13931,\n",
       " 15523,\n",
       " 5255,\n",
       " 8282,\n",
       " 12418,\n",
       " 4096,\n",
       " 13681,\n",
       " 6526,\n",
       " 9905,\n",
       " 6459,\n",
       " 4114,\n",
       " 17370,\n",
       " 12904,\n",
       " 2803,\n",
       " 16331,\n",
       " 2681,\n",
       " 753,\n",
       " 18196,\n",
       " 3383,\n",
       " 12869,\n",
       " 15523,\n",
       " 5255,\n",
       " 17621,\n",
       " 9564,\n",
       " 18245,\n",
       " 8277,\n",
       " 7306,\n",
       " 10846,\n",
       " 11905,\n",
       " 17370,\n",
       " 4247,\n",
       " 16808,\n",
       " 2687,\n",
       " 1057,\n",
       " 11905,\n",
       " 6912,\n",
       " 1596,\n",
       " 6685,\n",
       " 7714,\n",
       " 2724,\n",
       " 14851,\n",
       " 8241,\n",
       " 16647,\n",
       " 17089,\n",
       " 6752,\n",
       " 2964,\n",
       " 16312,\n",
       " 10244,\n",
       " 8945,\n",
       " 15555,\n",
       " 7660,\n",
       " 15694,\n",
       " 5255,\n",
       " 9564,\n",
       " 6752,\n",
       " 2964,\n",
       " 14179,\n",
       " 15563,\n",
       " 16675,\n",
       " 532,\n",
       " 16675,\n",
       " 10770,\n",
       " 16675,\n",
       " 16202,\n",
       " 16504,\n",
       " 8282,\n",
       " 12538,\n",
       " 3705,\n",
       " 11591,\n",
       " 12033,\n",
       " 6685,\n",
       " 15840,\n",
       " 7306,\n",
       " 12033,\n",
       " 4096,\n",
       " 11746,\n",
       " 15170,\n",
       " 15523,\n",
       " 5531,\n",
       " 10046,\n",
       " 14567,\n",
       " 2483,\n",
       " 6685,\n",
       " 14792,\n",
       " 12350,\n",
       " 15170,\n",
       " 14150,\n",
       " 13931,\n",
       " 9621,\n",
       " 6625,\n",
       " 8277,\n",
       " 13075,\n",
       " 9243,\n",
       " 2913,\n",
       " 2728,\n",
       " 6444,\n",
       " 15555,\n",
       " 12629,\n",
       " 15828,\n",
       " 12176,\n",
       " 2924,\n",
       " 16034,\n",
       " 7062,\n",
       " 12105]"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded_abstracts[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[    0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0     0     0     0     0     0     0     0     0     0     0     0\n",
      "     0  5255 11430 18245  8277 10852  3597  9893 12003  7599 15556  9474\n",
      "   340  8560 17163  5850 16064   641 16498  7749 15858 12350 11027 13936\n",
      "  2362 13230  4597  8311 14531 17644  4096  7651 18041  7980 17673 13931\n",
      " 15523  5255  8282 12418  4096 13681  6526  9905  6459  4114 17370 12904\n",
      "  2803 16331  2681   753 18196  3383 12869 15523  5255 17621  9564 18245\n",
      "  8277  7306 10846 11905 17370  4247 16808  2687  1057 11905  6912  1596\n",
      "  6685  7714  2724 14851  8241 16647 17089  6752  2964 16312 10244  8945\n",
      " 15555  7660 15694  5255  9564  6752  2964 14179 15563 16675   532 16675\n",
      " 10770 16675 16202 16504  8282 12538  3705 11591 12033  6685 15840  7306\n",
      " 12033  4096 11746 15170 15523  5531 10046 14567  2483  6685 14792 12350\n",
      " 15170 14150 13931  9621  6625  8277 13075  9243  2913  2728  6444 15555\n",
      " 12629 15828 12176  2924 16034  7062 12105]\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.keras import layers\n",
    "padded_abstracts = tf.keras.preprocessing.sequence.pad_sequences(encoded_abstracts,\n",
    "                                                              padding='pre')\n",
    "\n",
    "print(padded_abstracts[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "def abstracts_split(padded_abstracts, labels):\n",
    "    \"\"\"\n",
    "    Splits data into train, validation and test set with a 80, 10, 10 split.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    padded_abstracts: abstracts padded to the same sequence length\n",
    "    labels: labels correspoding to abstracts\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    train_x: Training data\n",
    "    train_y: Training labels\n",
    "    valid_x: Validation data\n",
    "    valid_y: Validation labels\n",
    "    test_x: Test data\n",
    "    test_y: Test labels\n",
    "    \"\"\"\n",
    "    train_x = padded_abstracts[: int(0.8 * len(padded_abstracts))]\n",
    "    train_y = labels[: int(0.8 * len(padded_abstracts))]\n",
    "    valid_x = padded_abstracts[\n",
    "        int(0.8 * len(padded_abstracts)) : int(0.9 * len(padded_abstracts))\n",
    "    ]\n",
    "    valid_y = labels[int(0.8 * len(padded_abstracts)) : int(0.9 * len(padded_abstracts))]\n",
    "    test_x = padded_abstracts[int(0.9 * len(padded_abstracts)) :]\n",
    "    test_y = labels[int(0.9 * len(padded_abstracts)) :]\n",
    "    return train_x, train_y, valid_x, valid_y, test_x, test_y\n",
    "\n",
    "\n",
    "def abstracts_create_dataloaders(\n",
    "    train_x, train_y, valid_x, valid_y, test_x, test_y, batch_size=50\n",
    "):\n",
    "    \"\"\"\n",
    "    Creates PyTorch data loaders\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    train_x: Training data\n",
    "    train_y: Training labels\n",
    "    valid_x: Validation data\n",
    "    valid_y: Validation labels\n",
    "    test_x: Test data\n",
    "    test_y: Test labels\n",
    "    batch_size: size of the batch\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    PyTorch data loaders for train, validation and test\n",
    "    \"\"\"\n",
    "    # create Tensor Dataset\n",
    "    train_data = TensorDataset(torch.LongTensor(train_x), torch.LongTensor(train_y))\n",
    "    valid_data = TensorDataset(torch.LongTensor(valid_x), torch.LongTensor(valid_y))\n",
    "    test_data = TensorDataset(torch.LongTensor(test_x), torch.LongTensor(test_y))\n",
    "\n",
    "    # dataloader\n",
    "    train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)\n",
    "    valid_loader = DataLoader(valid_data, batch_size=batch_size, shuffle=False)\n",
    "    test_loader = DataLoader(test_data, batch_size=batch_size, shuffle=False)\n",
    "    return train_loader, valid_loader, test_loader\n",
    "\n",
    "\n",
    "def abstracts_train(\n",
    "    net,\n",
    "    train_loader,\n",
    "    valid_loader,\n",
    "    lr=0.001,\n",
    "    epochs=3,\n",
    "    clip=5,\n",
    "    print_every=5,\n",
    "    criterion=nn.BCEWithLogitsLoss(),\n",
    "    device=\"cpu\",\n",
    "):\n",
    "    \"\"\"\n",
    "    Train a network on the abstracts data\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    net: Initialized model\n",
    "    train_loader: Dataloader containing the training data\n",
    "    valid_loader: Dataloader containing the validation data\n",
    "    lr: learning rate\n",
    "    epochs: number of epochs\n",
    "    clip: clip gradients at this value\n",
    "    print_every: print current train and validation loss every x steps\n",
    "    criterion: Loss function\n",
    "    device: device to train on - cpu or cuda\n",
    "    \"\"\"\n",
    "    optimizer = torch.optim.SGD(\n",
    "        [param for param in net.parameters() if param.requires_grad == True], lr=lr\n",
    "    )\n",
    "    batch_processed_counter = 0\n",
    "\n",
    "    net = net.to(device)\n",
    "    #criterion = criterion.to(device)\n",
    "\n",
    "    net.train()\n",
    "\n",
    "    # train for some number of epochs\n",
    "    for e in range(epochs):\n",
    "        valid_counter = 0\n",
    "        epoch_val_losses = []\n",
    "        losses = []\n",
    "        print(\"Starting epoch\", e + 1)\n",
    "\n",
    "        # batch loop\n",
    "        for inputs, labels in train_loader:\n",
    "            batch_processed_counter += 1\n",
    "            inputs = inputs.to(device)\n",
    "            labels = labels.to(device)\n",
    "\n",
    "            # zero accumulated gradients\n",
    "            net.zero_grad()\n",
    "\n",
    "            # get the output from the model\n",
    "            # output, h = net(inputs, h)\n",
    "            output = net(inputs)\n",
    "\n",
    "            # calculate the loss and perform backprop\n",
    "            loss = criterion(output.squeeze(), labels)\n",
    "            losses.append(loss.item())\n",
    "            loss.backward()\n",
    "            # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs /\n",
    "            # LSTMs.\n",
    "            nn.utils.clip_grad_norm_(net.parameters(), clip)\n",
    "            optimizer.step()\n",
    "\n",
    "            # loss stats\n",
    "            if batch_processed_counter % print_every == 0:\n",
    "                valid_counter += 1\n",
    "                # Get validation loss\n",
    "                val_losses = []\n",
    "                net.eval()\n",
    "                for inputs, labels in valid_loader:\n",
    "                    inputs = inputs.to(device)\n",
    "                    labels = labels.to(device)\n",
    "\n",
    "                    output = net(inputs)\n",
    "                    val_loss = criterion(output.squeeze(), labels)\n",
    "\n",
    "                    val_losses.append(val_loss.item())\n",
    "                    epoch_val_losses.append(val_loss.item())\n",
    "\n",
    "                net.train()\n",
    "                print(\n",
    "                    \"Epoch: {:2d}/{:2d}   \".format(e + 1, epochs),\n",
    "                    \"Batch: {:2d}  \".format(batch_processed_counter),\n",
    "                    \"Batch loss: {:.6f}   \".format(loss.item()),\n",
    "                    \"Val loss: {:.6f}\".format(np.mean(val_losses)),\n",
    "                )\n",
    "\n",
    "        print(\n",
    "            (\n",
    "                \"Finished epoch {}. Average batch loss: {}. \"\n",
    "                \"Average validation loss: {}\"\n",
    "            ).format(e + 1, np.mean(losses), np.mean(epoch_val_losses))\n",
    "        )\n",
    "\n",
    "def abstracts_test(net, test_loader, criterion=nn.BCELoss(), device=\"cpu\"):\n",
    "    \"\"\"\n",
    "    Evaluate a trained network on the review data.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    net: trained model\n",
    "    test_loader: Dataloader containing the training data\n",
    "    criterion: Loss function\n",
    "    device: device to train on - cpu or cuda\n",
    "    \"\"\"\n",
    "    test_losses = []  # track loss\n",
    "    num_correct = 0\n",
    "\n",
    "    net.eval()\n",
    "\n",
    "    net = net.to(device)\n",
    "    criterion = criterion.to(device)\n",
    "\n",
    "    # iterate over test data\n",
    "    for inputs, labels in test_loader:\n",
    "        inputs = inputs.to(device)\n",
    "        labels = labels.to(device)\n",
    "\n",
    "        output = net(inputs)\n",
    "\n",
    "        # calculate loss\n",
    "        test_loss = criterion(output.squeeze(), labels.float())\n",
    "        test_losses.append(test_loss.item())\n",
    "\n",
    "        # convert output probabilities to predicted class (0 or 1)\n",
    "        pred = torch.round(output.squeeze())  # rounds to the nearest integer\n",
    "\n",
    "        # compare predictions to true label\n",
    "\n",
    "        num_correct += (pred == labels).sum().item()\n",
    "\n",
    "    # -- stats! -- ##\n",
    "    # avg test loss\n",
    "    print(\"Test loss: {:.3f}\".format(np.mean(test_losses)))\n",
    "\n",
    "    # accuracy over all test data\n",
    "    test_acc = num_correct / len(test_loader.dataset)\n",
    "    print(\"Test accuracy: {:.3f}\".format(test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "677 85 85\n"
     ]
    }
   ],
   "source": [
    "# Split dataset into 80% training, 10% test, and 10% validation Dataset\n",
    "train_x, train_y, valid_x, valid_y, test_x, test_y = abstracts_split(\n",
    "    padded_abstracts, new_labels\n",
    ")\n",
    "print(len(train_y), len(valid_y), len(test_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0., 1., 0., 0., 0., 0., 0., 0., 0., 1.])"
      ]
     },
     "execution_count": 292,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.Tensor(train_y[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create data loaders for training\n",
    "train_loader, valid_loader, test_loader = abstracts_create_dataloaders(\n",
    "    train_x, train_y, valid_x, valid_y, test_x, test_y\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 294,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create an LSTM for sentiment analysis\n",
    "class SimpleLSTM(nn.Module):\n",
    "    \"\"\"\n",
    "    The RNN model that will be used to perform classification.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        vocab_size,\n",
    "        embedding_dim,\n",
    "        hidden_dim,\n",
    "        num_layers=1,\n",
    "        lstm_dropout_prob=0.5,\n",
    "        dropout_prob=0.3,\n",
    "    ):\n",
    "        \"\"\"\n",
    "        Initialize the model by setting up the layers\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        vocab_size: number of unique words in the abstracts\n",
    "        embeddings_dim: size of the embeddings\n",
    "        hidden_dim: dimension of the LSTM output\n",
    "        num_layers: number of LSTM layers\n",
    "        lstm_dropout_prob: dropout applied between the LSTM layers\n",
    "        dropout_prob: dropout applied before the fully connected layer\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "\n",
    "        self.num_layers = num_layers\n",
    "        self.hidden_dim = hidden_dim\n",
    "\n",
    "        self.emb = torch.nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.lstm = torch.nn.LSTM(embedding_dim, self.hidden_dim, self.num_layers, batch_first = True, \n",
    "                                  dropout = lstm_dropout_prob)\n",
    "        self.drop = torch.nn.Dropout(p=dropout_prob)\n",
    "        self.fc = torch.nn.Linear(self.hidden_dim,10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        Perform a forward pass of our model on some input and hidden state.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        x: batch as a (batch_size, sequence_length) tensor\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "       \n",
    "        \"\"\"\n",
    "        # init hidden layer, which is needed for the LSTM\n",
    "        batch_size = len(x)\n",
    "        hidden = self.init_hidden(batch_size)\n",
    "        \n",
    "        embeds = self.emb(x)\n",
    "        lstm_out, hidden = self.lstm(embeds, hidden)\n",
    "        lstm_out = lstm_out.contiguous().view(-1, self.hidden_dim)\n",
    "        \n",
    "        out = self.drop(lstm_out)\n",
    "        out = self.fc(out)\n",
    "        \n",
    "        #out = out.view(batch_size, -1)\n",
    "        #out = out[:,-1]\n",
    "        return out\n",
    "            \n",
    "\n",
    "    def init_hidden(self, batch_size):\n",
    "        \"\"\"\n",
    "        Initialize hidden state.\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        Empty hidden LSTM state.\n",
    "        \"\"\"\n",
    "\n",
    "        # Create two new tensors with sizes num_layers x batch_size x hidden_dim,\n",
    "        # initialized to zero, for hidden state and cell state of LSTM\n",
    "        weight = next(self.parameters())  # only used to determine device\n",
    "\n",
    "        hidden = (\n",
    "            weight.new(self.num_layers, batch_size, self.hidden_dim).zero_(),\n",
    "            weight.new(self.num_layers, batch_size, self.hidden_dim).zero_(),\n",
    "        )\n",
    "\n",
    "        return hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SimpleLSTM(\n",
      "  (emb): Embedding(18568, 100)\n",
      "  (lstm): LSTM(100, 100, batch_first=True, dropout=0.5)\n",
      "  (drop): Dropout(p=0.3, inplace=False)\n",
      "  (fc): Linear(in_features=100, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "vocab_size = len(word_ids) + 1  # +1 for the 0 padding\n",
    "embedding_dim = 100\n",
    "hidden_dim = 100\n",
    "num_layers = 1\n",
    "\n",
    "# this may raise a warning when num_layers=1 (which is fine)\n",
    "model = SimpleLSTM(vocab_size, embedding_dim, hidden_dim, num_layers)\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting epoch 1\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Target size (torch.Size([50, 10])) must be the same as input size (torch.Size([19550, 10]))",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-306-effc1a72904e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mn_epochs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m5\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mabstracts_train\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalid_loader\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepochs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mn_epochs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-290-721e5ba1a95a>\u001b[0m in \u001b[0;36mabstracts_train\u001b[1;34m(net, train_loader, valid_loader, lr, epochs, clip, print_every, criterion, device)\u001b[0m\n\u001b[0;32m    118\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    119\u001b[0m             \u001b[1;31m# calculate the loss and perform backprop\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 120\u001b[1;33m             \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    121\u001b[0m             \u001b[0mlosses\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    122\u001b[0m             \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\module.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    548\u001b[0m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    549\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 550\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    551\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    552\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\torch\\nn\\modules\\loss.py\u001b[0m in \u001b[0;36mforward\u001b[1;34m(self, input, target)\u001b[0m\n\u001b[0;32m    615\u001b[0m                                                   \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    616\u001b[0m                                                   \u001b[0mpos_weight\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpos_weight\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 617\u001b[1;33m                                                   reduction=self.reduction)\n\u001b[0m\u001b[0;32m    618\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    619\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\torch\\nn\\functional.py\u001b[0m in \u001b[0;36mbinary_cross_entropy_with_logits\u001b[1;34m(input, target, weight, size_average, reduce, reduction, pos_weight)\u001b[0m\n\u001b[0;32m   2431\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2432\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mtarget\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2433\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Target size ({}) must be the same as input size ({})\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtarget\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2434\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2435\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbinary_cross_entropy_with_logits\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mweight\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpos_weight\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreduction_enum\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Target size (torch.Size([50, 10])) must be the same as input size (torch.Size([19550, 10]))"
     ]
    }
   ],
   "source": [
    "# Fit and evaluate a model (without pretrained embeddings)\n",
    "n_epochs = 5\n",
    "\n",
    "abstracts_train(model, train_loader, valid_loader, epochs = n_epochs)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
